/*
 * Saphre - Suffix Arrays for Phrase Extraction
 * Copyright (C) 2013 
 * Dale Gerdemann Tübingen, Germany 
 * Niko Schenk Frankfurt am Main, Germany
 * All rights reserved.
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software
 * Foundation, either version 3 of the License, or (at your option) any later
 * version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program. If not, see <http://www.gnu.org/licenses/>.
 *
 */
package collectors.impl;

import util.sedgewick.StdOut;
import collectors.api.Collector1;
import util.Interval;
import saphre.core.Store;
import saphre.core.SuffixArray;
import util.sorting.Multiset;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;


/*
 * This program looks for extensions of a phrase to the left and
 * right. The idea is to use this program to filter out some phrases
 * that are incomplete. For example, "the right way" combines to the
 * right with "to" and "of" both with very low mutual information. The
 * combination to the left with "set" is higher, suggesting that "set
 * the right way" is a candidate larger phrase, however "set" only
 * occurs twice in this context, so it is perhaps rejected for being
 * under a threshold.
 *
 * <p>the right way:11
 * <p>r:to:2 3.52
 * <p>r:of:2 3.61
 * <p>l:set:2 9.38
 * <p>r:and:2 3.91
 * <p>l:is:2 4.34
 * <p>l:it:2 5.07
 * 
 * @author Dale Gerdemann
 *
 */
public class SupPhraseCollector implements Collector1 {

    Store store;
    SuffixArray sa;
    SuffixArray pa;
    double D;
    Interval top;
    int N;

    public SupPhraseCollector(Store store, SuffixArray sa, Interval top,
            int N) {
        this.store = store;
        D = (double) store.numDocs();
        this.sa = sa;
        this.top = top;
        this.N = N;
    }

    static double log2(double x) {
        return Math.log(x) / Math.log(2);
    }

    
    static class StringInt {

        String s;
        int id;
        int tf;
        double mi;
        boolean left;

        @Override
        public String toString() {
            return String.format("%4.2f   ", mi)
                    + (left ? "l" : "r") + ":" + s + ":" + tf;
        }
    }

    static class SIComp implements Comparator<StringInt> {

        @Override
        public int compare(StringInt a, StringInt b) {
            if (a.mi < b.mi) {
                return 1;
            } else if (a.mi > b.mi) {
                return -1;
            } else {
                return 0;
            }
        }

        public boolean equals(StringInt a, StringInt b) {
            if (a.mi == b.mi && a.tf == b.tf && a.left == b.left
                    && a.s.equals(b.s)) {
                return true;
            }
            return false;
        }
    }

    public void addPrefixArray(SuffixArray pa) {
    }

    @Override
    public void preAdd(Interval inter, int parentLcp, int lcp, int depth) {
    }

    @Override
    public void add(Interval inter, int lcp, int depth, Multiset docDist,
            Multiset leftContext) {
        if (lcp > 0 && lcp < 2 && leftContext.size() > 1) {
            List<Integer> lvals = new ArrayList<Integer>();
            List<Integer> l = leftContext.freqGreater(3, lvals);
            Multiset m = new Multiset();
            for (int i = inter.lb(); i <= inter.rb(); i++) {
                m.incr(store.text[sa.getSuftab()[i] + lcp]);
            }
            List<Integer> rvals = new ArrayList<Integer>();
            List<Integer> r = m.freqGreater(3, rvals);
            int loc = sa.getSuftab()[inter.lb()];

            if (l.size() + r.size() == 0) {
                //StdOut.println("Accept: "+store.toString(loc, loc+lcp)+":"+f1);
                //StdOut.println("\n");
            } else {

                int df = docDist.size();
                int tf = inter.tf();
                int f1 = tf;
                double observed = -Math.log(df / D);
                double predicted = Math.log(1 - Math.exp(-tf / D));
                double ridf = observed - predicted;

                StdOut.println(store.toString(loc, loc + lcp) + ":" + tf + ":"
                        + String.format(" _%3.1f_ ", ridf));

                List<StringInt> c = new ArrayList<StringInt>();
                for (int i = 0; i < l.size(); i++) {
                    StringInt k = new StringInt();
                    k.s = store.toString(l.get(i));
                    k.id = l.get(i);
                    k.tf = lvals.get(i);
                    k.left = true;
                    c.add(k);
                }
                for (int i = 0; i < r.size(); i++) {
                    StringInt k = new StringInt();
                    k.s = store.toString(r.get(i));
                    k.id = r.get(i);
                    k.tf = rvals.get(i);
                    k.left = false;
                    c.add(k);
                }

                for (StringInt si : c) {
                    int[] pat = new int[1];
                    pat[0] = si.id;
                    Interval found = sa.search(pat, top);
                    int f2 = found.tf();
                    int O = si.tf;
                    double E = f1 * f2 / (double) N;
                    double MI = log2(O / E);
                    si.mi = MI;
                }
                SIComp cmp = new SIComp();
                Collections.sort(c, cmp);
                for (StringInt si : c) {
                    StdOut.print(si.toString());
                    if (!(si.mi > 10 && si.left == false)) {
                        StdOut.println();
                    } else {
                        StdOut.print(" ==> ");
                        int[] pat = new int[lcp + 1];
                        int[] patPrefix = store.toArray(loc, loc + lcp);
                        for(int i = 0; i < patPrefix.length; i++) {
                            pat[i] = patPrefix[i];
                        }
                        pat[pat.length - 1] = si.id;
                        // StdOut.println(store.toString(pat));
                        Interval found = sa.search(pat, top);
                        int locf = sa.getSuftab()[found.lb()];
                        int lcpf = sa.getLcp(found);
                        Multiset docDistf = found.docDist(sa, store);
                        int dff = docDistf.size();
                        int tff = found.tf();
                        double observedf = -Math.log(dff / D);
                        double predictedf = Math.log(1 - Math.exp(-tff / D));
                        double ridff = observedf - predictedf;

                        StdOut.println(store.toString(locf, locf + lcpf)
                                + ":" + tff + ":"
                                + String.format(" _%3.1f_", ridff));
                        if (tff > tf) {
                            StdOut.println("!!!!!!!!!");
                        }
                        System.out.println(docDistf.toString(store.getDocNames(), "?"));
                    }
                }
                StdOut.println("\n");
            }

        }
    }

    @Override
    public void addTrivial(Interval inter, int lcp, int depth,
            Multiset docDist, Multiset leftContext) {
    }
}
